
package org.sopt.dev.ImgKnight.agent.Executor;

import java.io.BufferedInputStream;

import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Vector;
import java.util.concurrent.Callable;

import net.htmlparser.jericho.HTMLElementName;

import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.HeadMethod;
import org.apache.log4j.Logger;

import org.sopt.dev.ImgKnight.agent.Agent;
import org.sopt.dev.ImgKnight.agent.Filter.ExtentionFilter;
import org.sopt.dev.ImgKnight.agent.URL.Util.UrlUtil;

public class TakeUrl implements Callable<List<String>> {
	
	private URL url = null;

	private String countryCode = null;
	
	private String countryName = null;
	
	private Agent agent = null;
	
	private Logger serviceLogger = null;
	
	private Logger exceptionLogger = null;
	
	public TakeUrl(URL url, String countryCode, String countryName) {
		this.url = url;
		this.countryCode = countryCode;
		this.countryName = countryName;
		this.agent = Agent.getInstance();
		serviceLogger = Logger.getLogger("agent.service");
		exceptionLogger = Logger.getLogger("agent.exception");
	}
	
	@Override
	public List<String> call() {
		//long start = System.currentTimeMillis();
		String baseUrl = url.toString();
		HashMap<String, List<String>> elementsMap = null;
		GetMethod method = null;
		BufferedInputStream responseBodyStream = null;
		Vector<String> tagNames = new Vector<String>(3);
		tagNames.add(HTMLElementName.A);
		tagNames.add(HTMLElementName.IMG);
		tagNames.add(HTMLElementName.FRAME);
		try {
			String contentType = getContentType(url);
			if( contentType != null && !contentType.contains("text/html") ) {
				serviceLogger.info("Discard: "+url+", not allowd content-type: "+contentType);
				agent.onDiscardCrawl();
				return null;
			}
			
			method = agent.getHttpHandler().getHttpGetMethod(url);
			if( method == null ) {
				agent.onMissedCrawl();
				return null;
			}
			
			responseBodyStream = new BufferedInputStream(method.getResponseBodyAsStream());		
			elementsMap =agent.getHttpHandler()
				.getHttpElementsMapFromStreamByStreamedSource(responseBodyStream, tagNames);
			
			if( method.getPath().length() > 1 ) {
				String query = method.getQueryString();
				baseUrl = UrlUtil.resolve(url.toString(), method.getPath());
				if( query != null && query.length() > 0 )
					baseUrl += "?"+query;
			}
		} catch (Exception e) {
			exceptionLogger.warn("Unexpected exception: "+e+": "+baseUrl);
			agent.onMissedCrawl();
			return null;
		} finally {
			if( responseBodyStream != null )
				try { responseBodyStream.close(); } catch (IOException ignore) {}
			if( method != null ) method.releaseConnection();
		}
		
		List<String> elementsList = null;
		List<String> extractedUrl = null;
		List<String> extractedResource = null;
		for(String tag : tagNames) {
			elementsList = elementsMap.get(tag);
			if( tag.equals(HTMLElementName.A) )
				extractedUrl = extractUrlByProperty(baseUrl, elementsList, "href");
			else if( tag.equals(HTMLElementName.IMG) )
				extractedResource = extractSrc(baseUrl, elementsList);
			else if( tag.equals(HTMLElementName.FRAME) )
				extractedUrl.addAll(extractUrlByProperty(baseUrl, elementsList, "src"));
		}
		elementsMap.clear();
		
		String [] allowResourceType = agent.getAgentConfig().AllowResourceType;
		mergeResource(extractedUrl, extractedResource, allowResourceType);
		
		if( extractedResource != null && extractedResource.size() > 0 ) {
			ExtentionFilter fileExtensionFilter 
				= new ExtentionFilter(allowResourceType);
			extractedResource = fileExtensionFilter.list(extractedResource);
			int preFilterSize = extractedResource.size();
			if( agent.getAgentConfig().UsedSeenImageFilter )
				extractedResource = agent.getImgDupFilter().list(url.getHost(), extractedResource);
			int postFilterSize = extractedResource.size();
			agent.getImgDupFilter().addDuplicatedImgCount(preFilterSize-postFilterSize);
			if( extractedResource.size() > 0 ) {
				extractedResource.add(0, baseUrl);
				agent.startTakeResource(extractedResource, countryCode, countryName);
			}
		}	
		//long end = System.currentTimeMillis();
		//serviceLogger.info("Fetch "+(agent.getTakeUrlCount()+1)+": "+url);
		//serviceLogger.debug("Link: "+extractedUrl.size()+" extract");
		//serviceLogger.debug("Resource: "+extractedResource.size()+" extract");
		//serviceLogger.debug("Required time: " + (end-start) + " milli-second");
		return extractedUrl;
	}
	
	private String getContentType(URL url) {
		String contentType = "";
		HeadMethod headMethod = null;
		try {
			headMethod = agent.getHttpHandler().getHttpHeadMethod(url);
			if( headMethod == null ) {
				return null;
			}
			Header header = headMethod.getResponseHeader("Content-Type");
			if( header != null ) {
				contentType = header.getValue();
			}
		} catch (Exception e) {
		} finally {
			if( headMethod != null ) headMethod.releaseConnection();
		}
		return contentType;
	}
	
	private List<String> extractUrlByProperty(String baseUrl, List<String> elementList, String property) {
		List<String> extracted = new LinkedList<String>();
		HashSet<String> pushed = new HashSet<String>();
		String value = null;
		int end = -1;
		if( elementList != null ) {
			for(String element : elementList) {
				int begin = property.length()+1;
				int offset = element.indexOf(property);
				if( offset == -1 ) continue;
				char q = element.charAt(offset+begin);
				if( q == '\"' ) {
					begin++;
					end = element.indexOf("\"", offset+begin);
				}
				else if( q == '\'' ) {
					begin++;
					end = element.indexOf("'", offset+begin);
				}
				else {
					end = element.indexOf(">", offset+begin);
					if( end == -1 )
						end = element.indexOf(" ", offset+begin);
				}
				if( end == -1 ) {
					exceptionLogger.info("Extract exception: "+element);
					continue;
				}
				
				value = element.substring(offset+begin, end);
				if( value == null )
					continue;
				value = value.trim();
				if( value.startsWith("#") )
					continue;
				if( value.startsWith("javascript") )
					continue;
				if( !value.startsWith("http") ) {
					if( UrlUtil.isRelative(value) )
						value = UrlUtil.resolve(baseUrl, value);
					else {
						exceptionLogger.info("Unreconized format: "+value+" baseURL:"+baseUrl);
						continue;
					}
				}
				try {
					value = UrlUtil.normalize(value, false, true);
				} catch (MalformedURLException ignore) {
					continue;
				}
				if( value != null ) {
					if( !pushed.contains(value) ) {
						extracted.add(value);
						pushed.add(value);
					}
				}	
			}	
		}
		return extracted;
	}
	
	private List<String> extractSrc(String baseUrl, List<String> elementList) {
		List<String> extracted = new LinkedList<String>();
		HashSet<String> pushed = new HashSet<String>();
		String src = null;
		String property = "src";
		int end = -1;
		if( elementList != null ) {
			for(String element : elementList) {
				int begin = property.length()+1;
				int offset = element.indexOf(property);
				if( offset == -1 ) continue;
				char q = element.charAt(offset+begin);
				if( q == '\"' ) {
					begin++;
					end = element.indexOf("\"", offset+begin);
				}
				else if( q == '\'' ) {
					begin++;
					end = element.indexOf("'", offset+begin);
				}
				else {
					end = element.indexOf(">", offset+begin);
					if( end == -1 )
						end = element.indexOf(" ", offset+begin);
				}	
				if( end == -1 ) {
					exceptionLogger.info("Extract exception: "+element);
					continue;
				}

				src = element.substring(offset+begin, end);
				if( src == null )
					continue;
				src = src.trim();
				if( src.startsWith("#") )
					continue;
				if( !src.startsWith("http") ) {
					if( UrlUtil.isRelative(src) )
						src = UrlUtil.resolve(baseUrl, src);
					else {
						exceptionLogger.info("Unreconized format: "+src+" baseURL:"+baseUrl);
						continue;
					}
				}
				try {
					src = UrlUtil.normalize(src, false, true);
				} catch (MalformedURLException ignore) {
					continue;
				}
				if( src != null ) {
					if( !pushed.contains(src) ) {
						extracted.add(src);
						pushed.add(src);
					}
				}
			}
		}
		return extracted;
	}
	
	private void mergeResource(List<String> extractedUrl, 
			List<String> extractedResource, String[] allowResourceType) {
		if( extractedUrl != null && extractedUrl.size() > 0 ) {
			Iterator<String> iter = extractedUrl.iterator();
			String url = null;
			while( iter.hasNext() ) {
				url = iter.next();
				if( UrlUtil.isFileSpecified(url) ) {
					for(String ext : allowResourceType) {
						if( url.endsWith(ext) ) {
							iter.remove();
							extractedResource.add(url);
							break;
						}
					}
				}
				else {
					/*URL unknowntype = null;
					try {
						unknowntype = new URL(url);
					} catch (MalformedURLException e) {
						serviceLogger.info("Discard: "+url+", unknown content-type");
					}
					if( unknowntype != null ) {
						String contentType = getContentType(unknowntype);
						if( contentType != null && contentType.contains("image") ) {
							iter.remove();
							extractedResource.add(url);
						}
					}*/
				}
			}
		}
	}
	
	public String getCountryCode() {
		return countryCode;
	}

	public String getCountryName() {
		return countryName;
	}
	
}
